<!DOCTYPE html><html lang="zh-CN" data-theme="dark"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>promehteus+grafana | J.のblog</title><meta name="keywords" content="prometheus,grafana"><meta name="author" content="J."><meta name="copyright" content="J."><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#0d0d0d"><meta name="description" content="一、Promethes和zabbix对比   对比项 Prometheus zabbix Prometheus优势 zabbix优势    管理 二进制文件启动 LNMP+编译 轻量级server，便于迁移和维护    配置 配置文件 图形化 更好的支持自动化配置 成本低   client 丰富的client库 zabbix_agent自定义脚本 为各种中间件、应用提供专业的exporter，监控">
<meta property="og:type" content="article">
<meta property="og:title" content="promehteus+grafana">
<meta property="og:url" content="http://dycloudnavite.love/2020/02/14/promehteus+grafana/index.html">
<meta property="og:site_name" content="J.のblog">
<meta property="og:description" content="一、Promethes和zabbix对比   对比项 Prometheus zabbix Prometheus优势 zabbix优势    管理 二进制文件启动 LNMP+编译 轻量级server，便于迁移和维护    配置 配置文件 图形化 更好的支持自动化配置 成本低   client 丰富的client库 zabbix_agent自定义脚本 为各种中间件、应用提供专业的exporter，监控">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://dycloudnavite.love/img/prometheus.png">
<meta property="article:published_time" content="2020-02-14T08:42:00.000Z">
<meta property="article:modified_time" content="2022-09-09T08:53:34.841Z">
<meta property="article:author" content="J.">
<meta property="article:tag" content="prometheus">
<meta property="article:tag" content="grafana">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://dycloudnavite.love/img/prometheus.png"><link rel="shortcut icon" href="/dujie5416/img/touxiang.png"><link rel="canonical" href="http://dycloudnavite.love/2020/02/14/promehteus+grafana/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/dujie5416/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/dujie5416/',
  algolia: {"appId":"9GWB2VV7GN","apiKey":"dac56ec5b0b1f45281251ee59db4d637","indexName":"blog","hits":{"per_page":6},"languages":{"input_placeholder":"搜索文章","hits_empty":"找不到您查询的内容：${query}","hits_stats":"找到 ${hits} 条结果，用时 ${time} 毫秒"}},
  localSearch: undefined,
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: 'promehteus+grafana',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2022-09-09 16:53:34'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/xxx.css"><link rel="stylesheet" href="/css/universe.css"><link rel="stylesheet" href="/css/custom.css"><style type="text/css">#toggle-sidebar {bottom: 80px}</style><link rel="stylesheet" href="https://fastly.jsdelivr.net/gh/tzy13755126023/BLOG_SOURCE/css/function.min.css"><link rel="stylesheet" href="/css/rightMenu.css"><style type="text/css">#toggle-sidebar {left:100px}</style><!-- hexo injector head_end start --><link rel="stylesheet" href="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/clock.min.css" /><!-- hexo injector head_end end --><meta name="generator" content="Hexo 6.2.0"><link rel="alternate" href="/dujie5416/atom.xml" title="J.のblog" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/dujie5416/img/touxiang.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/dujie5416/archives/"><div class="headline">文章</div><div class="length-num">73</div></a><a href="/dujie5416/tags/"><div class="headline">标签</div><div class="length-num">17</div></a><a href="/dujie5416/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/dujie5416/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/dujie5416/img/longzhu.png')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/dujie5416/">J.のblog</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/dujie5416/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">promehteus+grafana</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2020-02-14T08:42:00.000Z" title="发表于 2020-02-14 16:42:00">2020-02-14</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2022-09-09T08:53:34.841Z" title="更新于 2022-09-09 16:53:34">2022-09-09</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/dujie5416/categories/prometheus/">prometheus</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="promehteus+grafana"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="一、Promethes和zabbix对比"><a href="#一、Promethes和zabbix对比" class="headerlink" title="一、Promethes和zabbix对比"></a>一、Promethes和zabbix对比</h1><table>
<thead>
<tr>
<th>对比项</th>
<th>Prometheus</th>
<th>zabbix</th>
<th>Prometheus优势</th>
<th>zabbix优势</th>
</tr>
</thead>
<tbody><tr>
<td>管理</td>
<td>二进制文件启动</td>
<td>LNMP+编译</td>
<td>轻量级server，便于迁移和维护</td>
<td></td>
</tr>
<tr>
<td>配置</td>
<td>配置文件</td>
<td>图形化</td>
<td>更好的支持自动化配置</td>
<td>成本低</td>
</tr>
<tr>
<td>client</td>
<td>丰富的client库</td>
<td>zabbix_agent自定义脚本</td>
<td>为各种中间件、应用提供专业的exporter，监控项更全面</td>
<td>支持自定义监控项，对监控涉及者的格局要求较高</td>
</tr>
<tr>
<td>数据存储方式</td>
<td>Prometheus TSDB</td>
<td>MySQL</td>
<td>监控数据以时间维度统计情况较多，时序数据库更实用与监控数据的次敦促，按时间索引性能更高</td>
<td>Mysql比较常用</td>
</tr>
<tr>
<td>数据处理</td>
<td>PromQl</td>
<td>MySQL</td>
<td>PromQL计算函数丰富，统计维度广</td>
<td>同上</td>
</tr>
<tr>
<td>二次开发</td>
<td>丰富的SDK</td>
<td>api</td>
<td>提供了Go、Java&#x2F;Scala、Python、Ruby等sdk二次开发更便捷</td>
<td>api适配较为常用</td>
</tr>
<tr>
<td>对云环境的支持</td>
<td>原生支持容器监控</td>
<td>更适合物理机监控</td>
<td>自动发现容器，更好的适配k8s</td>
<td></td>
</tr>
<tr>
<td>告警方式</td>
<td>可按照标签分组，收敛</td>
<td>在次数上收敛</td>
<td>告警收敛方式更多样化</td>
<td></td>
</tr>
<tr>
<td>监控项值</td>
<td>支持数字</td>
<td>支持数字字符串</td>
<td></td>
<td>可做日志监控</td>
</tr>
</tbody></table>
<h1 id="二、Prometheus工作流程："><a href="#二、Prometheus工作流程：" class="headerlink" title="二、Prometheus工作流程："></a>二、Prometheus工作流程：</h1><h4 id="1、服务端"><a href="#1、服务端" class="headerlink" title="1、服务端"></a>1、服务端</h4><blockquote>
<p>Prometheus服务端是以一个进程的方式启动，下载安装完成之后，只需要启动一个进程即可，程序默认监听9090端口。每次采集到的数据叫做metrics。这些采集到的数据会先存放到内存中，然后定期的写入磁盘，如果服务器重启会将硬盘数据写会内存，所以对内存有一定消耗，Pormetheus不需要重视历史数据，所以默认只会保留15天的数据。</p>
</blockquote>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# nohup .&#x2F;prometheus &amp;
[root@bogon prometheus]# ps -ef |grep prometheus
root       2620   2618  0 15:33 ?        00:00:04 .&#x2F;prometheus
root       6414   2631  0 16:01 pts&#x2F;1    00:00:00 grep prometheus
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>
<h4 id="2、客户端"><a href="#2、客户端" class="headerlink" title="2、客户端"></a>2、客户端</h4><p>Prometheus 客户端分为pull和push 两种方式。</p>
<ul>
<li>pull模式</li>
</ul>
<blockquote>
<p>如果pull形式的话是服务端主动向客户端拉取数据，这样需要客户端上安装exporter作为守护进程（官网提供了很多exporters可以下载使用），比如node_exporter，几乎把系统自身相关数据全部采集了，非常全面，node_exporter默认监听9100端口。</p>
</blockquote>
<ul>
<li>push模式</li>
</ul>
<blockquote>
<p>如果是push形式的话客户端需要安装pushgateway插件，然后需要运维人员用脚本吧监控数据以key&#x2F;value的形式提交给pushgateway，在由pushgateway提交给服务端。（这种模式适用于exporters无法满足监控需求时可以定义）</p>
</blockquote>
<h4 id="3、metrics主要数据类型"><a href="#3、metrics主要数据类型" class="headerlink" title="3、metrics主要数据类型"></a>3、metrics主要数据类型</h4><ul>
<li>Gauges</li>
</ul>
<blockquote>
<p>最简单、使用最多的指标，获取一个返回值，这个返回值没有变化规律，不能肯定他是按照一定增长或减少的状态，采集回来的是多少就是多少。比如硬盘容量、CPU内存使用率都适合使用Gauges类型</p>
</blockquote>
<ul>
<li>Counters</li>
</ul>
<blockquote>
<p>计数器。数据从0开始累积，理想状态下应该是永远增长或者是不变。适合统计机器开机时间、HTTP访问量。</p>
</blockquote>
<ul>
<li>Histograms</li>
</ul>
<blockquote>
<p>和summary一样属于高级指标，用于统计数据的分布情况。比如最小值、最大值、中间值。比如统计一天的日志，大部分用户响应时间都是正常的，只有少量用户异常，如果这使用取平均值的话，这少量用户的异常情况就会被掩盖，而Histograms可以分别统计出全部用户的响应时间，比如0-1秒的用户有多少、1-2秒的用户有多少。</p>
</blockquote>
<p>联邦集群监控架构图<br><img src="https://img-blog.csdnimg.cn/00bfa02b27c441e6a472f07bcb0f1182.png" alt="在这里插入图片描述"><br>每个数据中心部署单独的Prometheus Server，用于采集当前数据中心的监控数据。并由一个中心的Prometheus Server 负责聚合多个数据中心的监控数据。</p>
<p>联邦集群是每个Prometheus Server都包含一个用于获取当前实例中监控样本的接口&#x2F;federate。不管是Prometheus实例还是exporter实例都可以获取。</p>
<h1 id="三、Prometheus-服务端安装"><a href="#三、Prometheus-服务端安装" class="headerlink" title="三、Prometheus 服务端安装"></a>三、Prometheus 服务端安装</h1><h4 id="1、首先下载prometheus服务端"><a href="#1、首先下载prometheus服务端" class="headerlink" title="1、首先下载prometheus服务端"></a>1、首先下载prometheus服务端</h4><p><a target="_blank" rel="noopener" href="https://github.com/prometheus/prometheus/releases">https://github.com/prometheus/prometheus/releases</a><br>或者官网下载<br><a target="_blank" rel="noopener" href="https://prometheus.io/download/">https://prometheus.io/download/</a></p>
<h4 id="2、解压安装包可以看到以下目录结构"><a href="#2、解压安装包可以看到以下目录结构" class="headerlink" title="2、解压安装包可以看到以下目录结构"></a>2、解压安装包可以看到以下目录结构</h4><pre class="line-numbers language-none"><code class="language-none">[root@bogon local]# cd prometheus&#x2F;
[root@bogon prometheus]# ll
总用量 95028
-rw-r--r--  1 root root        0 12月  4 12:57 1000
drwxr-xr-x. 2 3434 3434     4096 11月  6 2018 console_libraries
drwxr-xr-x. 2 3434 3434     4096 11月  6 2018 consoles
drwxr-xr-x. 3 root root     4096 12月  6 11:44 data
-rw-r--r--  1 root root     1920 12月  2 17:34 go.go
-rw-r--r--. 1 3434 3434    11357 11月  6 2018 LICENSE
-rw-------  1 root root    62726 12月  6 16:00 nohup.out
-rw-r--r--. 1 3434 3434     2769 11月  6 2018 NOTICE
-rwxr-xr-x. 1 3434 3434 58074162 11月  6 2018 prometheus
-rw-r--r--  1 root root      767 12月  3 15:24 prometheus.sh
-rw-r--r--  1 3434 3434     1772 12月  6 09:31 prometheus.yml
-rwxr-xr-x. 1 3434 3434 39113694 11月  6 2018 promtool
drwxr-xr-x. 2 root root     4096 12月  6 10:43 rules<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<h4 id="3、运行promtheus程序"><a href="#3、运行promtheus程序" class="headerlink" title="3、运行promtheus程序"></a>3、运行promtheus程序</h4><pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# nohup .&#x2F;prometheus &amp;
<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>查看端口</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# netstat -tunlp |grep prometheus
tcp        0      0 :::9090                     :::*                        LISTEN      2620&#x2F;.&#x2F;prometheus <span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>在浏览器中输入server端ip地址：ip:9090<br><img src="https://img-blog.csdnimg.cn/556afae68c844819878f980c1950b6f6.png" alt="在这里插入图片描述"><br>可以查看当前监控的节点，因为没有监控其他client端，所以目前只监控了server本身<br><img src="https://img-blog.csdnimg.cn/8fb0889a99934c6b8917cd8bb1baadd2.png" alt="在这里插入图片描述"><br>只监控了prometheus服务端本身<br><img src="https://img-blog.csdnimg.cn/9ed455242c184736baeecb359d9bc698.png" alt="在这里插入图片描述"></p>
<h4 id="4、配置文件"><a href="#4、配置文件" class="headerlink" title="4、配置文件"></a>4、配置文件</h4><p>prometheus配置文件分四个配置</p>
<ul>
<li>global：全局配置</li>
<li>alerting：告警配置</li>
<li>rule_files：告警规则</li>
<li>scrape_configs：配置数据源（target）<br>prometheus 配置文件：</li>
</ul>
<blockquote>
<p>horbor_labels：配置为true可以确保当采集到的监控指标冲突时，能够自动忽略冲突的监控数据。如果为false时，prometheus会自动将冲突的标签替换为exported_ 的形式</p>
</blockquote>
<blockquote>
<p>match[]：指定需要获取的时间序列。match[]参数必须是一个瞬时向量选择器，比如：up或者{job&#x3D;”api-server”}。配置多个match[]参数，用于获取多组时间序列的监控数据。</p>
</blockquote>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# cat prometheus.yml
# my global config
global:
  scrape_interval:     15s # # 拉取targets 的默认时间间隔 Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # # 执行rules的时间间隔 Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
      - targets:
          - 172.16.5.25:9093

# Load rules once and periodically evaluate them according to the global &#39;evaluation_interval&#39;.
rule_files:  #告警规则
   # - &quot;first_rules.yml&quot;
   # - &quot;second_rules.yml&quot;
  - &#x2F;usr&#x2F;local&#x2F;prometheus&#x2F;rules&#x2F;all.yml   #告警规则文件路径
                 # A scrape configuration containing exactly one endpoint to scrape:
                 # Here it&#39;s Prometheus itself.

scrape_configs:
  # The job name is added as a label &#96;job&#x3D;&lt;job_name&gt;&#96; to any timeseries scraped from this config.
  - job_name: slaves
    honor_labels: true
    metrics_path: &#x2F;federate
    params:
      match[]:
        - &#39;&#123;__name__&#x3D;~&quot;^slave:.*&quot;&#125;&#39;   # Request all slave-level time series
        - &#39;&#123;__name__&#x3D;~&quot;job:.*&quot;&#125;&#39;
        - &#39;&#123;__name__&#x3D;~&quot;node.*&quot;&#125;&#39;

    static_configs:  #静态发现两个slave端（因为slave端就两台所以可以直接用静态发现）
      - targets:
        - 172.16.5.122:9090
        - 172.16.5.25:9090
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<h4 id="5、配置告警规则"><a href="#5、配置告警规则" class="headerlink" title="5、配置告警规则"></a>5、配置告警规则</h4><pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# cat rules&#x2F;all.yml 
groups:
    - name: 主机状态-监控告警
      rules:
      - alert: 主机状态
        expr: up &#x3D;&#x3D; 0
        for: 1m
        labels:
          status: 非常严重
        annotations:
          summary: &quot;&#123;&#123;$labels.instance&#125;&#125;:服务器宕机&quot;
          description: &quot;&#123;&#123;$labels.instance&#125;&#125;:服务器延时超过5分钟&quot;
      
      - alert: CPU使用情况
        expr: 100-(avg(irate(node_cpu_seconds_total&#123;mode&#x3D;&quot;idle&quot;&#125;[5m])) by(instance)* 100) &gt; 60
        for: 1m
        labels:
          status: 一般告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; CPU使用率过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125; CPU使用大于60%(目前使用:&#123;&#123;$value&#125;&#125;%)&quot;
  
      - alert: 内存使用
        expr: 100 -(node_memory_MemTotal_bytes -node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes ) &#x2F; node_memory_MemTotal_bytes * 100&gt; 80
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; 内存使用率过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125; 内存使用大于80%(目前使用:&#123;&#123;$value&#125;&#125;%)&quot;
      - alert: IO性能
        expr: 100-(avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) &lt; 60
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; 流入磁盘IO使用率过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125; 流入磁盘IO大于60%(目前使用:&#123;&#123;$value&#125;&#125;)&quot;
 
      - alert: 网络
        expr: ((sum(rate (node_network_receive_bytes_total&#123;device!~&#39;tap.*|veth.*|br.*|docker.*|virbr*|lo*&#39;&#125;[5m])) by (instance)) &#x2F; 100) &gt; 102400
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; 流入网络带宽过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125;流入网络带宽持续2分钟高于100M. RX带宽使用率&#123;&#123;$value&#125;&#125;&quot;
 
      - alert: 网络
        expr: ((sum(rate (node_network_transmit_bytes_total&#123;device!~&#39;tap.*|veth.*|br.*|docker.*|virbr*|lo*&#39;&#125;[5m])) by (instance)) &#x2F; 100) &gt; 102400
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; 流出网络带宽过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125;流出网络带宽持续2分钟高于100M. RX带宽使用率&#123;&#123;$value&#125;&#125;&quot;
      
      - alert: TCP会话
        expr: node_netstat_Tcp_CurrEstab &gt; 1000
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; TCP_ESTABLISHED过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125; TCP_ESTABLISHED大于1000%(目前使用:&#123;&#123;$value&#125;&#125;%)&quot;
 
      - alert: 磁盘容量
        expr: 100-(node_filesystem_free_bytes&#123;fstype&#x3D;~&quot;ext4|xfs&quot;&#125;&#x2F;node_filesystem_size_bytes &#123;fstype&#x3D;~&quot;ext4|xfs&quot;&#125;*100) &gt; 70
        for: 1m
        labels:
          status: 严重告警
        annotations:
          summary: &quot;&#123;&#123;$labels.mountpoint&#125;&#125; 磁盘分区使用率过高！&quot;
          description: &quot;&#123;&#123;$labels.mountpoint &#125;&#125; 磁盘分区使用大于80%(目前使用:&#123;&#123;$value&#125;&#125;%)&quot;
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<h1 id="四、安装alertmanager-告警插件"><a href="#四、安装alertmanager-告警插件" class="headerlink" title="四、安装alertmanager 告警插件"></a>四、安装alertmanager 告警插件</h1><h4 id="1、安装下载alertmanager"><a href="#1、安装下载alertmanager" class="headerlink" title="1、安装下载alertmanager"></a>1、安装下载alertmanager</h4><p>下载地址：<br><a target="_blank" rel="noopener" href="https://github.com/prometheus/alertmanager">https://github.com/prometheus/alertmanager</a><br>解压之后文件结构：</p>
<pre class="line-numbers language-none"><code class="language-none">[root@client local]# cd alertmanager&#x2F;
[root@client alertmanager]# ll
总用量 49440
-rwxr-xr-x. 1 3434 3434 27610164 9月   3 23:02 alertmanager
-rw-r--r--. 1 root root      676 12月  6 10:52 alertmanager.yml
-rw-r--r--. 1 3434 3434      380 9月   3 23:37 alertmanager.yml.bak
-rw-r--r--. 1 root root     1359 12月  6 09:45 alertmanager.yml.youxiang
-rwxr-xr-x. 1 3434 3434 22935908 9月   3 23:02 amtool
drwxr-xr-x. 2 root root     4096 12月  6 17:36 data
-rw-r--r--. 1 3434 3434    11357 9月   3 23:37 LICENSE
-rw-------. 1 root root    37468 12月  6 17:20 nohup.out
-rw-r--r--. 1 3434 3434      457 9月   3 23:37 NOTICE<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>启动alertmanager</p>
<pre class="line-numbers language-none"><code class="language-none">[root@client alertmanager]# nohup .&#x2F;alertmanager &amp;
[1] 22490
nohup: 忽略输入并把输出追加到&quot;nohup.out&quot;<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>
<p>查看进程</p>
<pre class="line-numbers language-none"><code class="language-none">[root@client alertmanager]# ps -ef |grep alertmanager
root       3334      1  0 15:36 ?        00:00:08 .&#x2F;alertmanager
root      22662  22093  0 17:45 pts&#x2F;1    00:00:00 grep alertmanager<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>
<h4 id="2、alertmanager配置文件"><a href="#2、alertmanager配置文件" class="headerlink" title="2、alertmanager配置文件"></a>2、alertmanager配置文件</h4><ul>
<li>邮箱报警配置文件</li>
</ul>
<pre class="line-numbers language-none"><code class="language-none">[root@client alertmanager]# cat alertmanager.yml
global:
  # resolve_timeout：解析超时时间
  resolve_timeout: 5m
  #     # smtp_smarthost: 使用email打开服务配置
  smtp_smarthost: &#39;smtp.163.com:465&#39;
  # smtp_from：指定通知报警的邮箱
  smtp_from: &#39;xxx@163.com&#39;
  # smtp_auth_username：邮箱用户名
  smtp_auth_username: &#39;xxx@163.com&#39;
  # smtp_auth_password：授权密码
  smtp_auth_password: &#39;SHAbidaohaode123&#39;
  # smtp_require_tls：是否启用tls
  smtp_require_tls: false
  # route标记：告警如何发送分配
route:
  # group_by：采用哪个标签作为分组的依据
  group_by: [&#39;alertname&#39;]
  # group_wait：分组等待的时间
  group_wait: 10s
  # group_interval：上下两组发送告警的间隔时间
  group_interval: 10s
  # repeat_interval：重复发送告警时间。默认1h
  repeat_interval: 1m
  # receiver 定义谁来通知报警
  receiver: &#39;mail&#39;
 
  # receiver标记：告警接受者 
receivers:
  # name：报警来源自定义名称
- name: &#39;mail&#39;
  # email_configs：通过邮箱发送报警
  email_configs:
  # to：指定接收端email
    - to: &#39;xxx@163.com&#39;
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>

<ul>
<li>企业微信报警</li>
</ul>
<pre class="line-numbers language-none"><code class="language-none">[root@client alertmanager]# cat alertmanager.yml.wechat 
global:
  resolve_timeout: 5m
  wechat_api_corp_id: &#39;wwf21db505fce2dde4&#39;  
  wechat_api_url: &#39;https:&#x2F;&#x2F;qyapi.weixin.qq.com&#x2F;cgi-bin&#x2F;&#39;
  wechat_api_secret: &#39;VmgazzYOKCeUe7JkdeKE_Je-PSIsZN85dHXaFQvFJME&#39;
 
route:
  group_by: [&#39;alertname&#39;]  #报警分组依据
  group_wait: 10s  #组等待时间
  group_interval: 10s  #发送前等待时间
  repeat_interval: 1h  #重复周期
  receiver: &#39;wechat&#39;   #默认报警接受者
receivers:
- name: &#39;wechat&#39;
  wechat_configs:
  - send_resolved: true
    to_party: &#39;1&#39;
    agent_id: &#39;1000002&#39;
    corp_id: &#39;xxxx&#39;
    api_url: &#39;https:&#x2F;&#x2F;qyapi.weixin.qq.com&#x2F;cgi-bin&#x2F;&#39;
    api_secret: &#39;xxxx-PSIsZN85dHXaFQvFJME&#39;
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>然后在server端配置文件中填写alerting</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# vim prometheus.yml 
alerting:
  alertmanagers:
  - static_configs:
    - targets: [&quot;172.16.5.25:9093&quot;]<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<h1 id="五、添加slave节点"><a href="#五、添加slave节点" class="headerlink" title="五、添加slave节点"></a>五、添加slave节点</h1><p>在两台slave节点安装prometheus server端，用来分布式监控</p>
<h4 id="1、启动prometheus"><a href="#1、启动prometheus" class="headerlink" title="1、启动prometheus"></a>1、启动prometheus</h4><p>解压完成放到后台启动prometheus</p>
<pre class="line-numbers language-none"><code class="language-none">[root@client node_exporter]# nohup .&#x2F;prometheus &amp;
<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>启动之后查看进程已经启动</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# ps -ef |grep prometheus
root      68158   2755  1 17:44 pts&#x2F;1    00:00:02 .&#x2F;prometheus
root      68725   2755  0 17:48 pts&#x2F;1    00:00:00 grep prometheus<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>
<h4 id="2、添加文件自动发现"><a href="#2、添加文件自动发现" class="headerlink" title="2、添加文件自动发现"></a>2、添加文件自动发现</h4><p>在prometheus目录下创建targets&#x2F;nodes目录并创建nodes.json文件用来自动发现被监控节点</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon nodes]# pwd
&#x2F;usr&#x2F;local&#x2F;prometheus&#x2F;targets&#x2F;nodes
[root@bogon nodes]# ll
总用量 4
-rw-r--r-- 1 root root 152 12月  9 17:44 nodes.json
#targetes 里面写的就是需要监控的机器，在prometheus配置文件直接调用这个文件就可以不用每次都手动改配置文件了
[root@bogon nodes]# cat nodes.json 
[&#123;
  &quot;targets&quot;: [
    &quot;172.16.5.25:9100&quot;,
    &quot;172.16.5.122:9100&quot;,
    &quot;172.16.1.32:9100&quot;
  ],
  &quot;labels&quot;: &#123;
    &quot;server&quot;: &quot;server&#x3D;172.16.5.122&quot;
  &#125;
&#125;]

<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<h4 id="3、slave端配置文件"><a href="#3、slave端配置文件" class="headerlink" title="3、slave端配置文件"></a>3、slave端配置文件</h4><pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# cat prometheus.yml
# my global config
global:
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).
  external_labels:
    slave: 1  # This is the 1nd slave. This prevents clashes between slaves.

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
      - targets:
          - 172.16.5.25:9093

            # Load rules once and periodically evaluate them according to the global &#39;evaluation_interval&#39;.
rule_files:
  - &#x2F;usr&#x2F;local&#x2F;prometheus&#x2F;rules&#x2F;all.yml  #指定报警规则文件路径
    # A scrape configuration containing exactly one endpoint to scrape:
    # Here it&#39;s Prometheus itself.
scrape_configs:
      # The job name is added as a label &#96;job&#x3D;&lt;job_name&gt;&#96; to any timeseries scraped from this config.
  - job_name: &#39;prometheus-172.16.5.122&#39;   #增加一个节点，取名为prometheus-172.16.5.122
    # Add usual service discovery here, such as static_configs
    metrics_path: &quot;&#x2F;metrics&quot;   #获取数据的路径 http:&#x2F;&#x2F;slaveip:9100&#x2F;metrics
    file_sd_configs:  #这个配置表示通过文件发现
    - files: 
      - targets&#x2F;nodes&#x2F;nodes.json # 读取目录下nodes.json文件
      refresh_interval: 1m  # 刷新频率，1 分钟
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>报警规则跟上面相同</p>
<blockquote>
<p>地址规范：<br>我这里server端地址：<br>172.16.5.30<br>两个slave地址分别为<br>172.16.5.122<br>172.16.5.25<br>因为只开了3台虚拟机，所以两台slave 监控的节点是各自本身和对方的ip<br>172.16.5.122额外监控了172.16.1.32运维平台的机器</p>
</blockquote>
<p>下面看图：<br>这是172.16.5.122 的某一个监控值<br><img src="https://img-blog.csdnimg.cn/a7106a17954c4800886c14b31973ca68.png" alt="在这里插入图片描述"><br>这是172.16.5.25的监控值<br><img src="https://img-blog.csdnimg.cn/8b90cd90509f4acf9c84cad879ed3628.png" alt="在这里插入图片描述"><br>这是server端监控的值，可以看到server端可以收集到两台slave的所有监控信息</p>
<h1 id="六、安装grafana图形显示工具"><a href="#六、安装grafana图形显示工具" class="headerlink" title="六、安装grafana图形显示工具"></a>六、安装grafana图形显示工具</h1><h4 id="1、安装启动"><a href="#1、安装启动" class="headerlink" title="1、安装启动"></a>1、安装启动</h4><p>下载地址：</p>
<pre class="line-numbers language-none"><code class="language-none">wget https:&#x2F;&#x2F;dl.grafana.com&#x2F;oss&#x2F;release&#x2F;grafana-6.5.1-1.x86_64.rpm
yum localinstall grafana-6.5.1-1.x86_64.rpm<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>启动grafana-server</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# service grafana-server start <span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>查看进程</p>
<pre class="line-numbers language-none"><code class="language-none">[root@bogon prometheus]# ps -ef |grep grafana
grafana    5316      1  0 08:58 ?        00:00:05 &#x2F;usr&#x2F;sbin&#x2F;grafana-server --pidfile&#x3D;&#x2F;var&#x2F;run&#x2F;grafana-server.pid --config&#x3D;&#x2F;etc&#x2F;grafana&#x2F;grafana.ini --packaging&#x3D;rpm cfg:default.paths.provisioning&#x3D;&#x2F;etc&#x2F;grafana&#x2F;provisioning cfg:default.paths.data&#x3D;&#x2F;var&#x2F;lib&#x2F;grafana cfg:default.paths.logs&#x3D;&#x2F;var&#x2F;log&#x2F;grafana cfg:default.paths.plugins&#x3D;&#x2F;var&#x2F;lib&#x2F;grafana&#x2F;plugins
root       7355   2750  0 09:16 pts&#x2F;1    00:00:00 grep grafana<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span></span></code></pre>
<p>浏览器中访问ip:3000<br><img src="https://img-blog.csdnimg.cn/5c871f7b4dda4eab996099b097c371b1.png" alt="在这里插入图片描述"></p>
<h4 id="2、添加数据源"><a href="#2、添加数据源" class="headerlink" title="2、添加数据源"></a>2、添加数据源</h4><p><img src="https://img-blog.csdnimg.cn/33969fd6c549487188d29b4c7e0ed1ce.png" alt="在这里插入图片描述"><br><img src="https://img-blog.csdnimg.cn/a8624be165a848d2a10dc549a334eef2.png" alt="在这里插入图片描述"><br>URL地址是prometheus-server<br><img src="https://img-blog.csdnimg.cn/506441696e8842a39c4201db72db5525.png" alt="在这里插入图片描述"><br>填写完成之后添加Save&amp;test，出现下图则添加成功<br><img src="https://img-blog.csdnimg.cn/0855ba0c8d8b43d0a426d252e23c5b8b.png" alt="在这里插入图片描述"></p>
<h4 id="3、导入dashboard模板"><a href="#3、导入dashboard模板" class="headerlink" title="3、导入dashboard模板"></a>3、导入dashboard模板</h4><p><img src="https://img-blog.csdnimg.cn/13eebbdda90b410bb76cde44042045d4.png" alt="在这里插入图片描述"><br>常用的prometheus 模板有8919和1860，然后将Data Source 选择为刚才添加的Prometheus数据源<br><img src="https://img-blog.csdnimg.cn/2c2a9d918b3a4e808bf851d6c10f1830.png" alt="在这里插入图片描述"><br>整体的监控图形显示<br><img src="https://img-blog.csdnimg.cn/94cb1b103bcd49149c3e4727f08228df.png" alt="在这里插入图片描述"></p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="http://dycloudnavite.love">J.</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://dycloudnavite.love/2020/02/14/promehteus+grafana/">http://dycloudnavite.love/2020/02/14/promehteus+grafana/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://dycloudnavite.love" target="_blank">J.のblog</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/dujie5416/tags/prometheus/">prometheus</a><a class="post-meta__tags" href="/dujie5416/tags/grafana/">grafana</a></div><div class="post_share"><div class="addthis_inline_share_toolbox"></div><script src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-5d034d84b42927bf" async="async"></script></div></div><div class="post-reward"><div class="reward-button"><i class="fas fa-qrcode"></i> 打赏</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="/dujie5416/img/wechat.jpg" target="_blank"><img class="post-qr-code-img" src="/dujie5416/img/wechat.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="/dujie5416/img/alipay.jpg" target="_blank"><img class="post-qr-code-img" src="/dujie5416/img/alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/dujie5416/2020/05/02/pipeline%E5%88%86%E5%B8%83%E5%BC%8F%E6%9E%84%E5%BB%BA%E5%92%8Cpipline%E5%9F%BA%E7%A1%80%E8%AF%AD%E6%B3%95/"><img class="prev-cover" src="/dujie5416/img/jenkins.png" onerror="onerror=null;src='/dujie5416/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">pipeline分布式构建和pipline基础语法</div></div></a></div><div class="next-post pull-right"><a href="/dujie5416/2019/10/10/centos7%E7%94%9F%E6%88%90%E9%9A%8F%E6%9C%BA%E6%95%B0/"><img class="next-cover" src="/dujie5416/img/centos.png" onerror="onerror=null;src='/dujie5416/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">生成随机数</div></div></a></div></nav><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div></div><div class="comment-wrap"><div><div id="lv-container" data-id="city" data-uid="MTAyMC81Njk5OS8zMzQ2Mw=="></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/dujie5416/img/touxiang.png" onerror="this.onerror=null;this.src='/dujie5416/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">J.</div><div class="author-info__description"></div></div><div class="card-info-data site-data is-center"><a href="/dujie5416/archives/"><div class="headline">文章</div><div class="length-num">73</div></a><a href="/dujie5416/tags/"><div class="headline">标签</div><div class="length-num">17</div></a><a href="/dujie5416/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%80%E3%80%81Promethes%E5%92%8Czabbix%E5%AF%B9%E6%AF%94"><span class="toc-number">1.</span> <span class="toc-text">一、Promethes和zabbix对比</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Prometheus%E5%B7%A5%E4%BD%9C%E6%B5%81%E7%A8%8B%EF%BC%9A"><span class="toc-number">2.</span> <span class="toc-text">二、Prometheus工作流程：</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1%E3%80%81%E6%9C%8D%E5%8A%A1%E7%AB%AF"><span class="toc-number">2.0.0.1.</span> <span class="toc-text">1、服务端</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2%E3%80%81%E5%AE%A2%E6%88%B7%E7%AB%AF"><span class="toc-number">2.0.0.2.</span> <span class="toc-text">2、客户端</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3%E3%80%81metrics%E4%B8%BB%E8%A6%81%E6%95%B0%E6%8D%AE%E7%B1%BB%E5%9E%8B"><span class="toc-number">2.0.0.3.</span> <span class="toc-text">3、metrics主要数据类型</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%89%E3%80%81Prometheus-%E6%9C%8D%E5%8A%A1%E7%AB%AF%E5%AE%89%E8%A3%85"><span class="toc-number">3.</span> <span class="toc-text">三、Prometheus 服务端安装</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1%E3%80%81%E9%A6%96%E5%85%88%E4%B8%8B%E8%BD%BDprometheus%E6%9C%8D%E5%8A%A1%E7%AB%AF"><span class="toc-number">3.0.0.1.</span> <span class="toc-text">1、首先下载prometheus服务端</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2%E3%80%81%E8%A7%A3%E5%8E%8B%E5%AE%89%E8%A3%85%E5%8C%85%E5%8F%AF%E4%BB%A5%E7%9C%8B%E5%88%B0%E4%BB%A5%E4%B8%8B%E7%9B%AE%E5%BD%95%E7%BB%93%E6%9E%84"><span class="toc-number">3.0.0.2.</span> <span class="toc-text">2、解压安装包可以看到以下目录结构</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3%E3%80%81%E8%BF%90%E8%A1%8Cpromtheus%E7%A8%8B%E5%BA%8F"><span class="toc-number">3.0.0.3.</span> <span class="toc-text">3、运行promtheus程序</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#4%E3%80%81%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-number">3.0.0.4.</span> <span class="toc-text">4、配置文件</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#5%E3%80%81%E9%85%8D%E7%BD%AE%E5%91%8A%E8%AD%A6%E8%A7%84%E5%88%99"><span class="toc-number">3.0.0.5.</span> <span class="toc-text">5、配置告警规则</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%9B%9B%E3%80%81%E5%AE%89%E8%A3%85alertmanager-%E5%91%8A%E8%AD%A6%E6%8F%92%E4%BB%B6"><span class="toc-number">4.</span> <span class="toc-text">四、安装alertmanager 告警插件</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1%E3%80%81%E5%AE%89%E8%A3%85%E4%B8%8B%E8%BD%BDalertmanager"><span class="toc-number">4.0.0.1.</span> <span class="toc-text">1、安装下载alertmanager</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2%E3%80%81alertmanager%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-number">4.0.0.2.</span> <span class="toc-text">2、alertmanager配置文件</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%94%E3%80%81%E6%B7%BB%E5%8A%A0slave%E8%8A%82%E7%82%B9"><span class="toc-number">5.</span> <span class="toc-text">五、添加slave节点</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1%E3%80%81%E5%90%AF%E5%8A%A8prometheus"><span class="toc-number">5.0.0.1.</span> <span class="toc-text">1、启动prometheus</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2%E3%80%81%E6%B7%BB%E5%8A%A0%E6%96%87%E4%BB%B6%E8%87%AA%E5%8A%A8%E5%8F%91%E7%8E%B0"><span class="toc-number">5.0.0.2.</span> <span class="toc-text">2、添加文件自动发现</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3%E3%80%81slave%E7%AB%AF%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-number">5.0.0.3.</span> <span class="toc-text">3、slave端配置文件</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%85%AD%E3%80%81%E5%AE%89%E8%A3%85grafana%E5%9B%BE%E5%BD%A2%E6%98%BE%E7%A4%BA%E5%B7%A5%E5%85%B7"><span class="toc-number">6.</span> <span class="toc-text">六、安装grafana图形显示工具</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1%E3%80%81%E5%AE%89%E8%A3%85%E5%90%AF%E5%8A%A8"><span class="toc-number">6.0.0.1.</span> <span class="toc-text">1、安装启动</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2%E3%80%81%E6%B7%BB%E5%8A%A0%E6%95%B0%E6%8D%AE%E6%BA%90"><span class="toc-number">6.0.0.2.</span> <span class="toc-text">2、添加数据源</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3%E3%80%81%E5%AF%BC%E5%85%A5dashboard%E6%A8%A1%E6%9D%BF"><span class="toc-number">6.0.0.3.</span> <span class="toc-text">3、导入dashboard模板</span></a></li></ol></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/07/29/%E6%A0%B9%E6%8D%AErouter%E6%98%BE%E7%A4%BA%E5%B7%A6%E4%BE%A7%E8%8F%9C%E5%8D%95%E6%A0%8F%E9%9A%90%E8%97%8F%E6%9F%90%E4%B8%AA%E8%8F%9C%E5%8D%95%E9%80%89%E9%A1%B9/" title="vue3左侧菜单隐藏某选项"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="vue3左侧菜单隐藏某选项"/></a><div class="content"><a class="title" href="/dujie5416/2023/07/29/%E6%A0%B9%E6%8D%AErouter%E6%98%BE%E7%A4%BA%E5%B7%A6%E4%BE%A7%E8%8F%9C%E5%8D%95%E6%A0%8F%E9%9A%90%E8%97%8F%E6%9F%90%E4%B8%AA%E8%8F%9C%E5%8D%95%E9%80%89%E9%A1%B9/" title="vue3左侧菜单隐藏某选项">vue3左侧菜单隐藏某选项</a><time datetime="2023-07-29T06:32:00.000Z" title="发表于 2023-07-29 14:32:00">2023-07-29</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/07/13/Vue%E7%8A%B6%E6%80%81%E7%AE%A1%E7%90%86/" title="Vue状态管理"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="Vue状态管理"/></a><div class="content"><a class="title" href="/dujie5416/2023/07/13/Vue%E7%8A%B6%E6%80%81%E7%AE%A1%E7%90%86/" title="Vue状态管理">Vue状态管理</a><time datetime="2023-07-13T09:29:00.000Z" title="发表于 2023-07-13 17:29:00">2023-07-13</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/27/%E5%9F%BA%E4%BA%8EIOC%E8%A7%A3%E5%86%B3%E5%AF%B9%E8%B1%A1%E4%BE%9D%E8%B5%96%E9%97%AE%E9%A2%98/" title="基于IOC解决对象依赖问题"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="基于IOC解决对象依赖问题"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/27/%E5%9F%BA%E4%BA%8EIOC%E8%A7%A3%E5%86%B3%E5%AF%B9%E8%B1%A1%E4%BE%9D%E8%B5%96%E9%97%AE%E9%A2%98/" title="基于IOC解决对象依赖问题">基于IOC解决对象依赖问题</a><time datetime="2023-06-27T03:32:00.000Z" title="发表于 2023-06-27 11:32:00">2023-06-27</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/23/Gin%E6%A1%86%E6%9E%B6/" title="Gin框架"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="Gin框架"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/23/Gin%E6%A1%86%E6%9E%B6/" title="Gin框架">Gin框架</a><time datetime="2023-06-23T06:32:00.000Z" title="发表于 2023-06-23 14:32:00">2023-06-23</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/04/ServeMux%E3%80%81httprouter/" title="ServeMux、httprouter"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="ServeMux、httprouter"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/04/ServeMux%E3%80%81httprouter/" title="ServeMux、httprouter">ServeMux、httprouter</a><time datetime="2023-06-04T03:32:00.000Z" title="发表于 2023-06-04 11:32:00">2023-06-04</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('/dujie5416/img/longzhu.png')"><div id="footer-wrap"><div class="copyright">&copy;2019 - 2024  <i id="heartbeat" class="fa fas fa-heartbeat"></i> J.</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a><a href="http://beian.miit.gov.cn/"  style="color:#f72b07" target="_blank">京ICP备2022023567号</a></div></div><link rel="stylesheet" href="https://fastly.jsdelivr.net/gh/HCLonely/images@master/others/heartbeat.min.css"></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="algolia-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">搜索</span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="search-wrap"><div id="algolia-search-input"></div><hr/><div id="algolia-search-results"><div id="algolia-hits"></div><div id="algolia-pagination"></div><div id="algolia-info"><div class="algolia-stats"></div><div class="algolia-poweredBy"></div></div></div></div></div><div id="search-mask"></div></div><div id="rightMenu"><div class="rightMenu-group rightMenu-small"><div class="rightMenu-item" id="menu-backward"><i class="fa-solid fa-arrow-left"></i></div><div class="rightMenu-item" id="menu-forward"><i class="fa-solid fa-arrow-right"></i></div><div class="rightMenu-item" id="menu-refresh"><i class="fa-solid fa-arrow-rotate-right"></i></div><div class="rightMenu-item" id="menu-home"><i class="fa-solid fa-house"></i></div></div><div class="rightMenu-group rightMenu-line rightMenuOther"><a class="rightMenu-item menu-link" href="/archives/"><i class="fa-solid fa-archive"></i><span>文章归档</span></a><a class="rightMenu-item menu-link" href="/categories/"><i class="fa-solid fa-folder-open"></i><span>文章分类</span></a><a class="rightMenu-item menu-link" href="/tags/"><i class="fa-solid fa-tags"></i><span>文章标签</span></a></div><div class="rightMenu-group rightMenu-line rightMenuNormal"><a class="rightMenu-item menu-link" id="menu-radompage" href="/random/index.html"><i class="fa-solid fa-shoe-prints"></i><span>随便逛逛</span></a><div class="rightMenu-item" id="menu-translate"><i class="fa-solid fa-earth-asia"></i><span>繁简切换</span></div><div class="rightMenu-item" id="menu-darkmode"><i class="fa-solid fa-moon"></i><span>切换模式</span></div></div></div><div id="rightmenu-mask"></div><div><script src="/dujie5416/js/utils.js"></script><script src="/dujie5416/js/main.js"></script><script src="/dujie5416/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/algoliasearch/dist/algoliasearch-lite.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instantsearch.js/dist/instantsearch.production.min.js"></script><script src="/dujie5416/js/search/algolia.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"><script>function loadLivere () {
  if (typeof LivereTower === 'object') {
    window.LivereTower.init()
  }
  else {
    (function(d, s) {
        var j, e = d.getElementsByTagName(s)[0];
        if (typeof LivereTower === 'function') { return; }
        j = d.createElement(s);
        j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
        j.async = true;
        e.parentNode.insertBefore(j, e);
    })(document, 'script');
  }
}

if ('Livere' === 'Livere' || !false) {
  if (false) btf.loadComment(document.getElementById('lv-container'), loadLivere)
  else loadLivere()
}
else {
  function loadOtherComment () {
    loadLivere()
  }
}</script></div><script defer src="https://npm.elemecdn.com/jquery@latest/dist/jquery.min.js"></script><script data-pjax defer src="https://npm.elemecdn.com/tzy-blog/lib/js/theme/chocolate.js"></script><canvas id="universe"></canvas><script defer src="/js/universe.js"></script><script defer data-pjax src="/js/rightMenu.js"></script><div class="aplayer no-destroy" data-id="7427714271" data-server="netease" data-type="playlist" data-fixed="true" data-mini="true" data-listFolded="false" data-order="random" data-lrctype="1" data-preload="none" data-autoplay="true" muted></div><script id="click-show-text" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/click-show-text.min.js" data-mobile="false" data-text="I,LOVE,YOU" data-fontsize="15px" data-random="false" async="async"></script><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/metingjs/dist/Meting.min.js"></script><script src="https://cdn.jsdelivr.net/npm/pjax/pjax.min.js"></script><script>let pjaxSelectors = ["meta[property=\"og:image\"]","meta[property=\"og:title\"]","meta[property=\"og:url\"]","head > title","#config-diff","#body-wrap","#rightside-config-hide","#rightside-config-show",".js-pjax"]

var pjax = new Pjax({
  elements: 'a:not([target="_blank"])',
  selectors: pjaxSelectors,
  cacheBust: false,
  analytics: false,
  scrollRestoration: false
})

document.addEventListener('pjax:send', function () {

  // removeEventListener scroll 
  window.tocScrollFn && window.removeEventListener('scroll', window.tocScrollFn)
  window.scrollCollect && window.removeEventListener('scroll', scrollCollect)

  typeof preloader === 'object' && preloader.initLoading()
  document.getElementById('rightside').style.cssText = "opacity: ''; transform: ''"
  
  if (window.aplayers) {
    for (let i = 0; i < window.aplayers.length; i++) {
      if (!window.aplayers[i].options.fixed) {
        window.aplayers[i].destroy()
      }
    }
  }

  typeof typed === 'object' && typed.destroy()

  //reset readmode
  const $bodyClassList = document.body.classList
  $bodyClassList.contains('read-mode') && $bodyClassList.remove('read-mode')

  typeof disqusjs === 'object' && disqusjs.destroy()
})

document.addEventListener('pjax:complete', function () {
  window.refreshFn()

  document.querySelectorAll('script[data-pjax]').forEach(item => {
    const newScript = document.createElement('script')
    const content = item.text || item.textContent || item.innerHTML || ""
    Array.from(item.attributes).forEach(attr => newScript.setAttribute(attr.name, attr.value))
    newScript.appendChild(document.createTextNode(content))
    item.parentNode.replaceChild(newScript, item)
  })

  GLOBAL_CONFIG.islazyload && window.lazyLoadInstance.update()

  typeof chatBtnFn === 'function' && chatBtnFn()
  typeof panguInit === 'function' && panguInit()

  // google analytics
  typeof gtag === 'function' && gtag('config', '', {'page_path': window.location.pathname});

  // baidu analytics
  typeof _hmt === 'object' && _hmt.push(['_trackPageview',window.location.pathname]);

  typeof loadMeting === 'function' && document.getElementsByClassName('aplayer').length && loadMeting()

  // prismjs
  typeof Prism === 'object' && Prism.highlightAll()

  typeof preloader === 'object' && preloader.endLoading()
})

document.addEventListener('pjax:error', (e) => {
  if (e.request.status === 404) {
    pjax.loadUrl('/404.html')
  }
})</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><!-- hexo injector body_end start --><script data-pjax>
  function butterfly_clock_anzhiyu_injector_config(){
    var parent_div_git = document.getElementsByClassName('sticky_layout')[0];
    var item_html = '<div class="card-widget card-clock"><div class="card-glass"><div class="card-background"><div class="card-content"><div id="hexo_electric_clock"><img class="entered loading" id="card-clock-loading" src="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/loading.gif" style="height: 120px; width: 100%;" data-ll-status="loading"/></div></div></div></div></div>';
    console.log('已挂载butterfly_clock_anzhiyu')
    if(parent_div_git) {
      parent_div_git.insertAdjacentHTML("afterbegin",item_html)
    }
  }
  var elist = 'null'.split(',');
  var cpage = location.pathname;
  var epage = 'all';
  var qweather_key = '56289903f0d448dfbe8262dba0175f70';
  var gaud_map_key = '01603f2b2d08d0e947bb101e9d9b148b';
  var baidu_ak_key = 'undefined';
  var flag = 0;
  var clock_rectangle = '112.982279,28.19409';
  var clock_default_rectangle_enable = 'false';

  for (var i=0;i<elist.length;i++){
    if (cpage.includes(elist[i])){
      flag++;
    }
  }

  if ((epage ==='all')&&(flag == 0)){
    butterfly_clock_anzhiyu_injector_config();
  }
  else if (epage === cpage){
    butterfly_clock_anzhiyu_injector_config();
  }
  </script><script src="https://widget.qweather.net/simple/static/js/he-simple-common.js?v=2.0"></script><script data-pjax src="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/clock.min.js"></script><!-- hexo injector body_end end --><script src="/dujie5416/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"tagMode":false,"debug":false,"model":{"jsonPath":"/dujie5416/live2dw/assets/tororo.model.json"},"display":{"position":"right","width":150,"height":350,"hOffset":20,"vOffset":-20},"mobile":{"show":false},"log":false,"pluginJsPath":"lib/","pluginModelPath":"assets/","pluginRootPath":"live2dw/"});</script><script async>window.onload=function(){var a=document.createElement('script'),b=document.getElementsByTagName('script')[0];a.type='text/javascript',a.async=!0,a.src='/sw-register.js?v='+Date.now(),b.parentNode.insertBefore(a,b)};</script></body></html>